
import pandas as pd
import numpy as np

from sklearn import metrics
from sklearn.model_selection import cross_val_score, KFold, StratifiedShuffleSplit, ShuffleSplit

# Pytorch
import torch
from torch.autograd import Variable
import torch.nn as nn 
import torchvision.transforms as transforms

# Visualisation libraries

## Progress Bar
import progressbar

## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output

## plotly
from plotly.offline import init_notebook_mode, iplot 
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px

## seaborn
import seaborn as sns
sns.set_style("whitegrid")
sns.set_context("paper", rc={"font.size":12,"axes.titlesize":14,"axes.labelsize":12})

## matplotlib
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
from matplotlib.patches import Ellipse, Polygon
import matplotlib.gridspec as gridspec
import matplotlib.colors
from pylab import rcParams
from matplotlib.font_manager import FontProperties
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
plt.style.use('seaborn-whitegrid')
import matplotlib as mpl
mpl.rcParams['figure.figsize'] = (17, 6)
mpl.rcParams['axes.labelsize'] = 14
mpl.rcParams['xtick.labelsize'] = 12
mpl.rcParams['ytick.labelsize'] = 12
mpl.rcParams['text.color'] = 'k'
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")


# loading data
train_data = pd.read_csv("digit_recognizer/train.csv", dtype = np.float32)
display(train_data.head())


# y set: Labels
y = train_data['label'].values
# X set: Pixels
X = train_data.iloc[:,1:].values/255


fig, ax = plt.subplots(nrows=4, ncols=12, figsize=(12, 6), subplot_kw={'xticks': [], 'yticks': []})
ax = ax.ravel()

font = FontProperties()
font.set_family('fantasy')

n = int(np.sqrt(X[0].shape)[0])
for i in range(len(ax)):
    _ = ax[i].imshow(X[i].reshape(n,n))
    _ = ax[i].set_title('%i' % y[i], fontproperties=font, fontsize = 16)
    _ = ax[i].set_aspect(1)
    
plt.tight_layout(h_pad=.02, w_pad=.2)


def Dist_Plot(y, Plot_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
                        specs=[[{"type": "table"},{"type": "pie"}]])
    
    # Right
    y = ToSeries(y)
    fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                         values= y.value_counts().values, pull=[0.1],
                         textfont=dict(size=Plot_dict['textfont']),
                         marker=dict(colors = Plot_dict['PieColors'],
                         line=dict(color='black', width=1))), row=1, col=2)
    fig.update_traces(hole=.5)
    fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
    
    # Left
    
    ## Table
    Table = y.value_counts().to_frame('Count').reset_index(drop = False)
    Table = Table.rename(columns = {'index':Plot_dict['legend_title']}).sort_values(by = [Plot_dict['legend_title']])
    Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()),2)

    T = Table.copy()
    T['Percentage'] = T['Percentage'].map(lambda x: '%% %.2f' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    TableColors = Plot_dict['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not Plot_dict['height'] == None:
        fig.update_layout(height = Plot_dict['height'])
    fig.show()
    
Plot_dict = dict(column_widths = [0.3, 0.7],  PieColors = None, textfont = 12,
                 TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Dist_Plot(y, Plot_dict)


Labels = np.unique(y).astype(int)

Test_Size = 0.3
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
    # X
    if isinstance(X, pd.DataFrame):
        X_train, X_test = X.loc[train_index], X.loc[test_index]
    else:
        X_train, X_test = X[train_index], X[test_index]
    # y    
    if isinstance(y, pd.Series):
        y_train, y_test = y[train_index], y[test_index]
    else:
        y_train, y_test = y[train_index], y[test_index]
del sss


def Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=3, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
                        specs=[[{"type": "table"},{'type':'domain'}, {'type':'domain'}]])
    
    y_train = ToSeries(y_train)
    y_test = ToSeries(y_test)
    # Right
    C = 2
    for y in [y_train, y_test]:
        fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                             values= y.value_counts().values, pull=[0.1],
                             textfont=dict(size=Plot_dict['textfont']),
                             marker=dict(colors = Plot_dict['PieColors'],
                             line=dict(color='black', width=1))), row=1, col=C)
        fig.update_traces(hole=.5)
        fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
        C+=1
    
    # Left
    
    # Table
    Table = pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
                       'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).astype(str)
    T = Table.copy()
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
        
    TableColors = Plot_dict['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not Plot_dict['height'] == None:
        fig.update_layout(height = Plot_dict['height'])
    fig.show()
    
Plot_dict = dict(column_widths = [0.3, 0.3, 0.3],  PieColors = None, textfont = 12,
                 TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict)


def Header(Text, L = 100, C = 'Blue', T = 'White'):
    BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN, 'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
         'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(BACK[C] + FORE[T] + Style.NORMAL + Text + Style.RESET_ALL + ' ' + FORE[C] +
          Style.NORMAL +  (L- len(Text) - 1)*'=' + Style.RESET_ALL)
    
def Line(L=100, C = 'Blue'):
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(FORE[C] + Style.NORMAL + L*'=' + Style.RESET_ALL)
    
def Search_List(Key, List): return [s for s in List if Key in s]

def TorchSets(Set):
    # Inut: Arrays
    # GPU Cuda
    if isinstance(Set, (pd.DataFrame, pd.Series)):
        Set = Set.values
    if torch.cuda.is_available():
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor).cuda())
        else:
            Out = Variable(torch.from_numpy(Set).cuda())
    # CPU
    else:
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor))
        else:
            Out = Variable(torch.from_numpy(Set))
    return Out

# Tensors
X_train_tensor = TorchSets(X_train)
y_train_tensor = TorchSets(y_train)
X_test_tensor = TorchSets(X_test)
y_test_tensor = TorchSets(y_test)
    
Batch_size = 100
iteration_number = int(4e4)

epochs_number = int(iteration_number / (len(X_train) / Batch_size))

# Pytorch train and test sets
Train_set = torch.utils.data.TensorDataset(X_train_tensor, y_train_tensor)
Test_set = torch.utils.data.TensorDataset(X_test_tensor, y_test_tensor)

# data loader
train_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)
test_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)


# Create MLP_Model
class MLP_Model(nn.Module):
    def __init__(self, input_Size, hidden_Size, output_Size):
        super(MLP_Model, self).__init__()
        # Linear function 1:
        self.fc1 = nn.Linear(input_Size, hidden_Size) 
        # Non-linearity 1
        self.relu1 = nn.ReLU()
        
        # Linear function 2:
        self.fc2 = nn.Linear(hidden_Size, int(hidden_Size/4))
        # Non-linearity 2
        self.tanh2 = nn.Tanh()
        
        # Linear function 3:
        self.fc3 = nn.Linear(int(hidden_Size/4), output_Size)  
    
    def forward(self, x):
        # Linear function 1
        out = self.fc1(x)
        # Non-linearity 1
        out = self.relu1(out)
        
        # Linear function 2
        out = self.fc2(out)
        # Non-linearity 2
        out = self.tanh2(out)
               
        # Linear function 4 (readout)
        out = self.fc3(out)
        return out
    
def Plot_history(history, Table_Rows = 25, yLim = 2):
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
                        specs=[[{"type": "scatter"},{"type": "table"}]])
    # Left
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Loss'].astype(float).values.round(4),
                             line=dict(color='OrangeRed', width= 1.5), name = 'Loss'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Accuracy'].astype(float).values,
                             line=dict(color='MidnightBlue', width= 1.5),  name = 'Accuracy'), 1, 1)
    fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
                  dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
                  legend_orientation='h')
    fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
                     showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    fig.update_yaxes(range=[0, yLim], showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    # Right
    ind = np.linspace(0, history.shape[0], Table_Rows, endpoint = False).round(0).astype(int)
    ind = np.append(ind, history.index[-1])
    history = history[history.index.isin(ind)]
    T = history.copy()
    T[['Loss','Accuracy']] = T[['Loss','Accuracy']].applymap(lambda x: '%.4e' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color='darkslategray',
                                       fill_color='Navy', align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.4, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color=['Lavender', 'white', 'white']),
                                      align=['center', 'center'], font_size=12,height=20)), 1, 2)
    fig.show()


n_Inst = X_train.shape[1]
n_Att = X_train.shape[1]
    
input_Size, output_Size = n_Inst, len(Labels)
hidden_Size = 256

# model
model = MLP_Model(input_Size, hidden_Size, output_Size)

# GPU
if torch.cuda.is_available():
    model.cuda()

# Cross Entropy Loss 
CEL= nn.CrossEntropyLoss()

# Optimizer 
learning_rate = 1e-2
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# Traning the Model
Count = 0
Loss_list = []
Iteration_list = []
Accuracy_list = []
MSE_list = []
MAE_list = []
Steps = 10

Progress_Bar = progressbar.ProgressBar(maxval= int(iteration_number+iteration_number/10),
                                       widgets=[progressbar.Bar('=', '|', '|'), progressbar.Percentage()])
    
# print('---------------------------------------------------------')
for epoch in range(epochs_number):
    for i, (Xtr, ytr) in enumerate(train_loader):
        
        # Variables
        Xtr = Variable(Xtr.view(-1, n_Att))
        ytr = Variable(ytr)
        
        # Set all gradients to zero
        optimizer.zero_grad()
        
        # Forward
        Out = model(Xtr.float())
        
        # loss
        loss = CEL(Out, ytr.long())
        
        # Backward (Calculating the gradients)
        loss.backward()
        
        # Update parameters
        optimizer.step()
        
        Count += 1
        
        del Xtr, ytr
        
        # Predictions
        if Count % Steps == 0:
            # Calculate Accuracy         
            Correct, Total = 0, 0
            # Predictions
            for Xts, yts in test_loader: 
                Xts = Variable(Xts.view(-1, n_Att))
                
                # Forward
                Out = model(Xts.float())
                
                # The maximum value of Out
                Predicted = torch.max(Out.data, 1)[1]
                
                # Total number of yts
                Total += len(yts)
                
                # Total Correct predictions
                Correct += (Predicted == yts).sum()
            del Xts, yts
            # storing loss and iteration
            Loss_list.append(loss.data)
            Iteration_list.append(Count)
            Accuracy_list.append(Correct / float(Total))
            
        Progress_Bar.update(Count)

Progress_Bar.finish()

history = pd.DataFrame({'Iteration': np.array(Iteration_list),
                      'Loss': np.array([x.cpu().data.numpy() for x in Loss_list]),
                      'Accuracy': np.array([x.cpu().data.numpy() for x in Accuracy_list])})
del Loss_list, Iteration_list, Accuracy_list

|=========================================================================|100%


Plot_history(history, Table_Rows = 18, yLim = 1)


def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
    if n_splits == None:
        Titles = ['Train Set', 'Test Set']
    else:
        Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
    CM = [CM_Train, CM_Test]
    Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
    for i in range(2):
        fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
        fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
        _ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
                        linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
        _ = ax[0].set_title('Confusion Matrix');
        Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
        _ = sns.heatmap(Temp,
                        annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
                       linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
        _ = ax[1].set_title('Normalized Confusion Matrix');

        for a in ax:
            _ = a.set_xlabel('Predicted labels')
            _ = a.set_ylabel('True labels'); 
            _ = a.xaxis.set_ticklabels(Labels)
            _ = a.yaxis.set_ticklabels(Labels)
            _ = a.set_aspect(1)


# Train
y_pred = model(X_train_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Train = pd.DataFrame(metrics.classification_report(y_train, y_pred, target_names=Labels, output_dict=True)).T
CM_Train = metrics.confusion_matrix(y_train, y_pred)
# Test
y_pred = model(X_test_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Test = pd.DataFrame(metrics.classification_report(y_test, y_pred, target_names=Labels, output_dict=True)).T
CM_Test = metrics.confusion_matrix(y_test, y_pred)

Reports_Train = Reports_Train.reset_index().rename(columns ={'index': 'Train Set'})
Reports_Test = Reports_Test.reset_index().rename(columns ={'index': 'Test Set'})
                                                 
display(Reports_Train.style.hide_index().set_properties(**{'background-color': 'HoneyDew', 'color': 'Black'}).\
        set_properties(subset=['Train Set'], **{'background-color': 'SeaGreen', 'color': 'White'}))
display(Reports_Test.style.hide_index().set_properties(**{'background-color': 'Azure', 'color': 'Black'}).\
        set_properties(subset=['Test Set'], **{'background-color': 'RoyalBlue', 'color': 'White'}))

PD = dict(FS = (20, 8), annot_kws = 10, shrink = .6)
Confusion_Mat(CM_Train, CM_Test, PD = PD, n_splits = None)


test_data = pd.read_csv("digit_recognizer/test.csv", dtype = np.float32)
test_data = test_data.values/255

if torch.cuda.is_available():
    test_data_tensor = Variable(torch.from_numpy(test_data).cuda())
else:
    test_data_tensor = Variable(torch.from_numpy(test_data))

Pred = model(test_data_tensor).cpu().data.numpy()
Pred_Labels = pd.DataFrame({'ImageId': np.arange(1,test_data.shape[0]+1), 'Label':Pred.argmax(axis=1)})
# Saving the results
Pred_Labels.to_csv('digit_recognizer/digit_recognizer_sub_PyANN.csv', index=False)

Train Set	precision	recall	f1-score	support
0	0.995855	0.996888	0.996371	2892.000000
1	0.993299	0.994511	0.993904	3279.000000
2	0.992484	0.993502	0.992993	2924.000000
3	0.991784	0.990808	0.991296	3046.000000
4	0.990543	0.992281	0.991411	2850.000000
5	0.996970	0.990964	0.993958	2656.000000
6	0.994477	0.994820	0.994649	2896.000000
7	0.993815	0.990912	0.992361	3081.000000
8	0.989835	0.992968	0.991399	2844.000000
9	0.989779	0.990791	0.990285	2932.000000
accuracy	0.992857	0.992857	0.992857	0.992857
macro avg	0.992884	0.992844	0.992863	29400.000000
weighted avg	0.992861	0.992857	0.992857	29400.000000

Test Set	precision	recall	f1-score	support
0	0.982913	0.974194	0.978534	1240.000000
1	0.980919	0.987900	0.984397	1405.000000
2	0.965272	0.976057	0.970635	1253.000000
3	0.968266	0.958621	0.963419	1305.000000
4	0.965546	0.963175	0.964359	1222.000000
5	0.959964	0.947322	0.953601	1139.000000
6	0.973684	0.983884	0.978758	1241.000000
7	0.966012	0.968939	0.967474	1320.000000
8	0.959770	0.958983	0.959376	1219.000000
9	0.944356	0.945860	0.945107	1256.000000
accuracy	0.966905	0.966905	0.966905	0.966905
macro avg	0.966670	0.966494	0.966566	12600.000000
weighted avg	0.966899	0.966905	0.966886	12600.000000

Digit Recognizer Dataset¶

Data Description¶

Train and Test sets¶

Setting up Tensor Arrays¶

Modeling¶

Confusion Matrix¶

Predictions¶

References¶

	label	...
0	1.0	...
1	0.0	...
2	1.0	...
3	4.0	...
4	0.0	...

	label	...
0	1.0	...
1	0.0	...
2	1.0	...
3	4.0	...
4	0.0	...

	label	...
0	1.0	...
1	0.0	...
2	1.0	...
3	4.0	...
4	0.0	...